#使用异步携程的方式,完成对北京环境部列表的爬取
import sys
import os
sys.path.append(os.getcwd()+"\creeper") 
import asyncio #异步模块
import aiohttp #异步请求模块
from aiohttp.client import request
import aiofiles #异步文件读写
import csv  # CSV 读写
from aiocsv import AsyncReader, AsyncWriter #异步CSV
from lxml import etree #Xpath解析模块
from Mylib import mkdir

import pandas as pd
import os
import re
import base64

Folder_Path = "C:\\Users\\Kunling\\Desktop\\" + os.path.basename(__file__).split('.')[0]         #要拼接的文件夹及其完整路径，注意不要包含中文
SaveFile_Path =  Folder_Path + "\\Catalog"       #拼接后要保存的文件路径
SaveFile_Name = r'环境部一级目录.csv'              #合并后要保存的文件名

#页面URL基本地址（环境部）
url_base = "http://gzcx.sthjj.beijing.gov.cn/eportal/admin?moduleId=f0270681ba5545089012be244338c858&struts.portlet.mode=view&struts.portlet.action=/portlet/commonSearch!getDataList.action"

my_header = {
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36"
}

my_cookies = {
    "JSESSIONID":"AAA842A09BA5A45DE4A3B265778F6212",
    "__jsluid_h":"a1f4d25420ab393497eda00d3903a5a1", 
    "xylD7dkicltxS":"5l8DC9_1wehvS1iysI0w4hwFGAIeOzjM7pQxhl8Ax24GnEw7Np_..q2V4_0xJydXGj2x3Que4JURM2Fqi9nAksG",
    "yfx_c_g_u_id_10002881":"_ck21072811442913025558580550733",
    "yfx_f_l_v_t_10002881":"f_t_1627443869296__r_t_1628053905628__v_t_1628056149183__r_c_1"
}
#异步协程下载HTML模块
async def aiodownload(url,number):
    
    async with aiohttp.ClientSession(cookies = my_cookies) as session:
        async with session.post(url, data={"params":getParams(1,number)},headers = my_header) as resp:
            #解析数据
            print(await resp.text())

def getParams(start, end):
    data = '{"start":"0", "end":"10","sourceId":"0902fd30eaf04a109c3601389e49ecf4","county":"5biC57qn","ApplicationID":"","ProjName":"","sort":" ACCEPTTIME desc"}'
    jiami = data.encode()
    res = base64.b64encode(jiami)
    return res

async def main():
    tasks = [asyncio.create_task(aiodownload(url_base,10))]
    await asyncio.wait(tasks)
    
#执行代码
if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())